
# 加载各类库文件
import os
import sys
import sqlite3
import csv
import pandas as pd
from tqdm import tqdm
from datetime import datetime



os.chdir('/rds/general/ephemeral/user/yz1117/ephemeral')
MIN_START_DATE = '1999-01-01'
MAX_END_DATE = '2025-01-30'

'''
    date_range = pd.date_range(start=MIN_START_DATE, end=MAX_END_DATE, freq='MS').date
    statistics = {}
    for key in tqdm(companies, total =l_companies,mininterval=100):
    # for key in companies:
        statistics[key] = [[0]*3 for _ in range(320)]
'''

def save_statistics(statistics, save_dir = "/rds/general/user/yz1117/home/Hangyun_Work", filename="statistics1999-2002.csv"):
    save_fn = os.path.join(save_dir, filename)
    date_range = pd.date_range(start=MIN_START_DATE, end=MAX_END_DATE, freq='MS').date
    with open(file = save_fn, mode = 'w') as file:
        writer = csv.writer(file)
        # writer.writerow(['rcid', 'date','in_employee','out_employee','stay_employee'])    
        writer.writerow(['rcid','in_employee','out_employee','stay_employee'])    
        for key in tqdm(statistics, mininterval=100):
            for i, _ in enumerate(date_range):
                row = [key,statistics[key][i][0],statistics[key][i][1],statistics[key][i][2]]
                writer.writerow(row)


def process_chunk(rows):
    # 在这里处理每个chunk的数据
    print(rows[:5])  # 仅打印前五行以验证读取成功

def read_large_csv_in_chunks(file_path,filename, percentage=10):
    file_path = os.path.join(file_path, filename)
    total_rows = sum(1 for line in open(file_path)) - 1  # 减去表头
    chunk_size = total_rows // (100 // percentage)

    with open(file_path, mode='r') as file:
        reader = csv.reader(file)
        header = next(reader)  # 读取表头
        print(header)  # 打印表头以验证读取成功

        rows = []
        for idx, row in enumerate(reader):
            rows.append(row)
            if (idx + 1) % chunk_size == 0:
                process_chunk(rows)
                rows = []

        if rows:  # 如果还有剩余行，处理这些行
            process_chunk(rows)


def load_csv_to_dataframe(file_path):
    with open(file_path, mode='r') as file:
        reader = csv.reader(file)
        header = next(reader)  # 读取表头
        rows = [row for row in reader]

    # 将数据加载到DataFrame中
    df = pd.DataFrame(rows, columns=header)
    # 打印DataFrame的前几行
    print(df.head())
    # 打印DataFrame的描述性统计
    print(df.describe())
    return df

def main():
    fpath = "statisticsposition2002.csv"
    save_dir = "/rds/general/ephemeral/user/yz1117/ephemeral/aibot-tmp-data"
    file_path = os.path(save_dir, fpath)
    # read_large_csv_in_chunks(save_dir,fpath)
    df = load_csv_to_dataframe(file_path)


if __name__ == "__main__":
    main()
